# scratch 大时代1994-柳岸话又明

import urllib.request
from bs4 import BeautifulSoup
import ssl
import time

# Ignore SSL certificate errors
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE

# Ask for URL input
url = input('Enter a url address (default https://m.boquge.com/wapbook/90695_155773726.html): ')
if url == '':
    url = 'https://m.boquge.com/wapbook/90695_155773726.html'

# Set up headers to mimic a real browser
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'}

with open('大时代1994_柳岸花又明_index.txt', 'r', encoding='utf-8') as file, open('大时代1994_柳岸花又明.txt', 'a', encoding='utf-8') as outfile:
    for url in file:
        time.sleep(1)

        # Create a request object with headers
        req = urllib.request.Request(url, headers=headers)

        # Open the URL using the request object
        html = urllib.request.urlopen(req, context=ctx).read()

        # Parse the HTML with BeautifulSoup
        soup = BeautifulSoup(html, 'html.parser')

        title = soup.find('h1')
        content = []
        content_div = soup.find('div', id='cContent')
        paragraphs = content_div.find_all('p')
        for paragrash in paragraphs:
            content.append(paragrash.get_text())

        # print(title.getText())
        # print(content)

        # save the tile and the content to a file.
        filename = "大时代1994_柳岸花又明.txt"
        # with open(filename, 'a', encoding='utf-8') as file:
        outfile.write(title.getText())
        outfile.write('\n')
        for line in content:
            outfile.write('\t')
            outfile.write(line)
            outfile.write('\n')
